/*--------------------------------------------------------------------------------
	DESCRIPTION: Final merge of survey datasets to build harmonized master 
				 dataset

--------------------------------------------------------------------------------*/

/*-------------------------------------------------------------------------------
1)	Run all relevant .do files to clean datasets
-------------------------------------------------------------------------------*/

clear
do "$code_dir/data_creation/cleaning_afrobarometer.do"
do "$code_dir/data_creation/cleaning_americasbarometer.do"
do "$code_dir/data_creation/cleaning_arabarometer.do"
do "$code_dir/data_creation/cleaning_asiabarometer.do"
do "$code_dir/data_creation/cleaning_ess.do"
do "$code_dir/data_creation/cleaning_gallup.do"
do "$code_dir/data_creation/cleaning_ivs.do"
do "$code_dir/data_creation/cleaning_latinobarometer.do"
do "$code_dir/data_creation/cleaning_lits.do"
do "$code_dir/data_creation/cleaning_southasianbarometer.do"
do "$code_dir/data_creation/cleaning_wjp.do"

/*-------------------------------------------------------------------------------
2)	Merge all datasets to create harmonized dataset
-------------------------------------------------------------------------------*/

clear
use "$data_dir/clean/ivs_clean.dta" //ivs
append using "$data_dir/clean/gallup_clean.dta" //gallup
append using "$data_dir/clean/wjp_clean.dta" //wjp
append using "$data_dir/clean/ess_clean.dta" //ess
append using "$data_dir/clean/americasbarometer_clean.dta" //americas barometer

//lits
forvalues i = 1/4 {
	append using "$data_dir/clean/lits`i'_clean.dta"
}

//afrobarometer
forvalues i = 1/8 {
	append using "$data_dir/clean/afro`i'_clean.dta"
}

//latinobarometer
forvalues i = 1/23 {
	append using "$data_dir/clean/latino`i'_clean.dta"
}

//arabarometer
forvalues i = 1/7 {
	append using "$data_dir/clean/arab`i'_clean.dta"
}

//asiabarometer
forvalues i = 1/6 {
	append using "$data_dir/clean/asia`i'_clean.dta"
}

//southasiabarometer
forvalues i = 1/2 {
	append using "$data_dir/clean/southasia`i'_clean.dta"
}

drop iso2
rename S009 iso2

//merge in regional variables
merge m:1 ccode using "$data_dir/raw/ccode_with_regions.dta"
replace region = "Europe" if ccode=="XKX"
replace subregion = "Eastern Europe" if ccode=="XKX"
drop if _merge==2
drop _merge

drop country 
local varlist ccode country_short survey wave weight trust_govt trust_govt_dk_ref trust_justice trust_police trust_civil_service trust_local_govt trust_parties trust_parliament trust_others trust_military trust_religious_organizations trust_state govt_corrupt media_free feelings_income standard_living survey_year age yob cohort sex religion edu occupation employment_status income ideology marital urban born_country
foreach var of local varlist {
	label var `var' ""
}

order ccode country_short survey wave survey_year region subregion
rename country_short country
sort ccode survey wave survey_year

/*-------------------------------------------------------------------------------
3)	Final cleaning of variables and data
-------------------------------------------------------------------------------*/

local varlist yob sex marital religion edu employment_status occupation income urban age born_country
foreach var of local varlist {
	replace `var' = -99 if `var' == .
}

drop if age>=0 & age<15 //drop all observations less than 15 years old
replace age = floor(age)

replace yob = floor(yob)
replace yob = -99 if yob==2023
replace yob = -99 if missing(yob)

label drop urban
label define marital 1 "Married" 2 "Not married" 3 "Other" -99 "Missing"
label define edu 1 "Primary" 2 "Secondary" 3 "Tertiary" 4 "None" -99 "Missing"
label define employment_status 1 "Employed" 2 "Unemployed" 3 "Other" -99 "Missing"
label define urban 1 "Urban" 2 "Rural" -99 "Missing"
label define born_country 0 "Not born in country" 1 "Born in country" -99 "Missing"
label define religion 1 "Do not belong to a denomination" 2 "Buddhist" 3 "Jewish" 4 "Christian" 5 "Muslim" 6 "Other" 7 "Hindu" -99 "Missing"
label define occupation 1 "Managers" 2 "Professionals" 3 "Technicians and Associate Professionals" 4 "Clerical Support Workers" 5 "Service and Sales Workers" 6 "Skilled Agricultural, Forestry and Fishery Workers" 7 "Craft and Related Trades Workers" 8 "Plant and Machine Operators, and Assemblers" 9 "Elementary Occupations" 0 "Armed Forces Occupations" -99 "Missing"
label define income 1 "Band 1" 2 "Band 2" 3 "Band 3" 4 "Band 4" 5 "Band 5" 6 "Band 6" 7 "Band 7" 8 "Band 8" 9 "Band 9" 10 "Band 10" -99 "Missing"

label values marital marital
label values edu edu
label values employment_status employment_status
label values urban urban
label values religion religion
label values occupation occupation
label values income income

sort ccode survey wave survey_year yob

drop if survey_year>2023 & survey_year!=.
drop if age>99 & age!=.

compress

save "$data_dir/clean/harmonized_dataset.dta", replace
